package com.company.reptile;

import org.apache.log4j.Logger;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/*描述 网络爬虫  多线程
尝试爬取某个茶座前100页的内容
 * @authro szq
 * @time 2022/01/05*/
public class Reptile {


    private static Logger log = Logger.getLogger(Reptile.class);
    private static String rootPath = "http://www.hualongxiang.com/";
    private static List<String> urlPathList;
    private static volatile String visitedPaths = "";
    //    原子更新基本类型 记录爬取的链接个数
    private static AtomicInteger linkcount = new AtomicInteger(0);
    private static Object xlock = new Object();

    public static void main(String[] args) throws InterruptedException {

//        构造100个网址
        urlPathList = new ArrayList<>();
        for (Integer i = 1; i <= 100; i++) {
            urlPathList.add(rootPath + "chazuo/p" + i.toString());
        }

//        多线程处理
//        创建4个线程 分别去爬取网页
        MyThread myThread1 = new MyThread("t1");
        MyThread myThread2 = new MyThread("t2");
        MyThread myThread3 = new MyThread("t3");
        MyThread myThread4 = new MyThread("t4");
        MyThread myThread5 = new MyThread("t5");
        MyThread myThread6 = new MyThread("t6");

        myThread1.start();
        myThread2.start();
        myThread3.start();
        myThread4.start();
        myThread5.start();
        myThread6.start();

        myThread1.join();
        myThread2.join();
        myThread3.join();
        myThread4.join();
        myThread5.join();
        myThread6.join();

        String[] visitedPathArray = visitedPaths.split(";");

        for (String visitedPath : visitedPathArray
        ) {
            System.out.println(visitedPath);
        }
        System.out.println("共爬取" + visitedPathArray.length + "个网页！");

    }

    /*爬取网页源代码并处理
     * @param requestUrl String 请求地址
     * @return */
    public static void AccessHtml(String urlPath, String threadName) {
        String msg = getHtml(urlPath);
        msg = msg.replace("\n", "").replace("\t", "");
        // ?:0|1;  *:0|n  +:1|n+
        String pattern = "<tr class=\"bbs_tr3 simsun\".+?/tr>";
        Matcher m = Pattern.compile(pattern).matcher(msg);
        while (m.find()) {
//            for (int i = 0; i <= m.groupCount(); i++) {//这里 i<m.groupCount();改成i<=m.groupCount();
                String msg2 = m.group(0);
                // ?:0|1;  *:0|n  +:1|n
                String pattern2 = "<a.*?/a>";
                Matcher m2 = Pattern.compile(pattern2).matcher(msg2);
                while (m2.find()) {
                    String msg3 = m2.group(0);
//                    <a href="chazuo/15469178" name="readlink"
//                      id="a_ajax_15469178" class="subject_t fs14"
//                      title="在常州找一个好的中医，难吗？今日话题：说说你遇见过的好中医！" style="">
//                    <b><font color="#FF0000">在常州找一个好的中医，难吗？今日话题：说说你遇见过的好中医！</font></b></a>
                    String pattern3 = "<a href=\"(.*)\" name=\"readlink\".*title=\"(.*)\" .*/a>";
                    Matcher m3 = Pattern.compile(pattern3).matcher(msg3);
                    while (m3.find()) {
                        int nowlinkcount = linkcount.incrementAndGet();
                        System.out.println(threadName + "-----------" + String.valueOf(nowlinkcount)
                                + "-----------" + rootPath + m3.group(1)
                                + " -----" + m3.group(2));
                    }
                }
//            }
        }
    }

    /*发起http get请求获取网页源代码
     * @param requestUrl String 请求地址
     * @return String 该地址的返回HTML 字符串*/
    public static String getHtml(String urlPath) {

        String msg = "";

        HttpURLConnection connection = null;
        try {
            // 1. 得到访问地址的URL
            URL url = new URL(urlPath);
            // 2. 得到网络访问对象java.net.HttpURLConnection
            connection = (HttpURLConnection) url.openConnection();
//             伪装浏览器
//            connection.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows 2000)");
            connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0");
            connection.setRequestProperty("Charsert", "UTF-8"); //设置请求编码
            /* 3. 设置请求参数（过期时间，输入、输出流、访问方式），以流的形式进行连接 */
            // 设置是否向HttpURLConnection输出
            connection.setDoOutput(false);
            // 设置是否从httpUrlConnection读入
            connection.setDoInput(true);
            // 设置请求方式　默认为GET
            connection.setRequestMethod("GET");
            // 设置是否使用缓存
            connection.setUseCaches(true);
            // 设置此 HttpURLConnection 实例是否应该自动执行 HTTP 重定向
            connection.setInstanceFollowRedirects(true);
            // 设置超时时间
            connection.setConnectTimeout(3000);
            // 连接
            connection.connect();
            // 4. 得到响应状态码的返回值 responseCode
            int code = connection.getResponseCode();
            // 5. 如果返回值正常，数据在网络中是以流的形式得到服务端返回的数据

            if (code == 200) { // 正常响应
                // 从流中读取响应信息
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(connection.getInputStream(), "GBK"));
                String line = null;
                while ((line = reader.readLine()) != null) { // 循环从流中读取
                    msg += line;
                }
                /////////////////
//                InputStream input = connection.getInputStream();
////                    StringBuffer sb1= new StringBuffer();
//                int len = 0;
//                byte[] buffer = new byte[1024];
//                StringBuilder builder = new StringBuilder();
//                while ((len = input.read(buffer)) != -1) {
////                        sb1.append((char)ss);
//                    builder.append(new String(buffer, 0, len, "GBK"));
//                }
//                msg = builder.toString();
                /////////////////
                reader.close(); // 关闭流
            }
//            // 显示响应结果
//            log.info(msg);
        } catch (IOException e) {
            log.error("转发出错，错误信息：" + e.getLocalizedMessage() + ";" + e.getClass());
        } finally {
            // 6. 断开连接，释放资源
            if (null != connection) {
                try {
                    connection.disconnect();
                } catch (Exception e) {
                    log.info("connection 流关闭异常：" + e.getLocalizedMessage());
                }
            }
        }

        return msg;

    }

    public void Reptile() {

    }

    public static class MyThread extends Thread {
        private String urlPath;
        private String threadName;

        public MyThread(String threadName) {
            this.urlPath = urlPath;
            this.threadName = threadName;
        }

        @Override
        public void run() {
            // 爬取网页源代码并处理
            for (String urlPath : urlPathList
            ) {
                Boolean flag = false;
//                try {
//                    Thread.sleep(100);
//                } catch (InterruptedException e) {
//                    e.printStackTrace();
//                }
                if (visitedPaths.contains(urlPath + ";")) {
                    flag = false;
                } else {
//                    没有爬取过
//                  这里的操作没有原子性，防止重复爬取网页需要加个锁
                    synchronized (xlock) {
//                        try {
//                            Thread.sleep(100);
//                        } catch (InterruptedException e) {
//                            e.printStackTrace();
//                        }
//                        这里还要判断一下，类似单例模式
                        if (visitedPaths.contains(urlPath + ";")) {
                            flag = false;
                        } else {
                            flag = true;
                            visitedPaths = visitedPaths + urlPath + ";";
                        }
                    }
                }
                if (flag == true) {
                    AccessHtml(urlPath, threadName);
                }
            }
        }
    }


}